package com.zelix.analy;

import java.io.*;
import java.util.HashSet;

/**
 * 拦截数据去重
 */
public class analyText {
    public static void main(String[] args) throws IOException {
        //去重
        //reduce();
    }

    //去重排序
    public static void reduce() throws IOException {
        HashSet<String> strs = new HashSet<>();
        FileInputStream fis = new FileInputStream("D:\\workspace\\temp\\long.txt");
        BufferedReader br = new BufferedReader(new InputStreamReader(fis));
        String tmp;
        while ((tmp = br.readLine()) != null) {
            strs.add(tmp);
        }
        br.close();
        fis.close();


        FileOutputStream fos = new FileOutputStream("D:\\workspace\\temp\\lesslong.txt");
        OutputStreamWriter outWriter = new OutputStreamWriter(fos);
        BufferedWriter bw = new BufferedWriter(outWriter);
        Object[] arr = strs.stream().sorted().toArray();
        for (Object o : arr) {
            bw.write(o.toString() + "\r\n");
        }

        bw.close();
        outWriter.close();
        fos.close();
    }
}
